{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "543b6208-f18f-4eba-8f2f-195fb35c1093",
   "metadata": {},
   "source": [
    "# Alternative Notion of Complexity, Formula Length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8d1f0369-52f9-434a-95ed-850a84d1b167",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import re\n",
    "import json\n",
    "import spacy\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b4592ed2-1197-42b6-aad0-a1fe660bfefb",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = json.load(open('MathQA/train.json'))\n",
    "dev_data = json.load(open('MathQA/dev.json'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "617c6c46-7bd7-46c1-8438-406d8ca03e9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_200_idx = np.load('MathQA/dev_200_idx.npy')\n",
    "dev_data_200 = []\n",
    "for i in dev_200_idx: dev_data_200.append(dev_data[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "bb279abd-3b10-4970-b54b-a54d31ebaa11",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_formula_len_dist = [len(d['annotated_formula']) for d in train_data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "55732868-416e-4e5e-850f-385247e0f43c",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[94,\n",
       " 101,\n",
       " 43,\n",
       " 36,\n",
       " 14,\n",
       " 114,\n",
       " 59,\n",
       " 59,\n",
       " 35,\n",
       " 137,\n",
       " 25,\n",
       " 89,\n",
       " 67,\n",
       " 40,\n",
       " 16,\n",
       " 108,\n",
       " 78,\n",
       " 38,\n",
       " 97,\n",
       " 52,\n",
       " 162,\n",
       " 134,\n",
       " 54,\n",
       " 28,\n",
       " 86,\n",
       " 76,\n",
       " 97,\n",
       " 107,\n",
       " 52,\n",
       " 60,\n",
       " 56,\n",
       " 77,\n",
       " 53,\n",
       " 65,\n",
       " 53,\n",
       " 108,\n",
       " 85,\n",
       " 15,\n",
       " 10,\n",
       " 75,\n",
       " 11,\n",
       " 57,\n",
       " 67,\n",
       " 109,\n",
       " 50,\n",
       " 80,\n",
       " 52,\n",
       " 146,\n",
       " 52,\n",
       " 44,\n",
       " 71,\n",
       " 54,\n",
       " 14,\n",
       " 87,\n",
       " 134,\n",
       " 19,\n",
       " 32,\n",
       " 42,\n",
       " 81,\n",
       " 326,\n",
       " 70,\n",
       " 113,\n",
       " 64,\n",
       " 65,\n",
       " 14,\n",
       " 54,\n",
       " 18,\n",
       " 69,\n",
       " 56,\n",
       " 25,\n",
       " 42,\n",
       " 65,\n",
       " 30,\n",
       " 58,\n",
       " 44,\n",
       " 117,\n",
       " 93,\n",
       " 9,\n",
       " 79,\n",
       " 100,\n",
       " 37,\n",
       " 40,\n",
       " 47,\n",
       " 273,\n",
       " 11,\n",
       " 57,\n",
       " 40,\n",
       " 43,\n",
       " 24,\n",
       " 37,\n",
       " 27,\n",
       " 102,\n",
       " 39,\n",
       " 28,\n",
       " 67,\n",
       " 59,\n",
       " 38,\n",
       " 103,\n",
       " 401,\n",
       " 14,\n",
       " 122,\n",
       " 93,\n",
       " 42,\n",
       " 294,\n",
       " 54,\n",
       " 125,\n",
       " 100,\n",
       " 113,\n",
       " 245,\n",
       " 75,\n",
       " 91,\n",
       " 60,\n",
       " 47,\n",
       " 39,\n",
       " 36,\n",
       " 33,\n",
       " 31,\n",
       " 106,\n",
       " 32,\n",
       " 36,\n",
       " 73,\n",
       " 39,\n",
       " 102,\n",
       " 77,\n",
       " 63,\n",
       " 145,\n",
       " 32,\n",
       " 29,\n",
       " 58,\n",
       " 79,\n",
       " 28,\n",
       " 47,\n",
       " 90,\n",
       " 88,\n",
       " 68,\n",
       " 46,\n",
       " 115,\n",
       " 84,\n",
       " 65,\n",
       " 193,\n",
       " 21,\n",
       " 56,\n",
       " 13,\n",
       " 15,\n",
       " 42,\n",
       " 99,\n",
       " 62,\n",
       " 145,\n",
       " 40,\n",
       " 29,\n",
       " 62,\n",
       " 37,\n",
       " 48,\n",
       " 78,\n",
       " 144,\n",
       " 58,\n",
       " 26,\n",
       " 32,\n",
       " 41,\n",
       " 95,\n",
       " 38,\n",
       " 49,\n",
       " 23,\n",
       " 46,\n",
       " 67,\n",
       " 76,\n",
       " 25,\n",
       " 87,\n",
       " 45,\n",
       " 120,\n",
       " 10,\n",
       " 64,\n",
       " 69,\n",
       " 47,\n",
       " 76,\n",
       " 85,\n",
       " 113,\n",
       " 37,\n",
       " 165,\n",
       " 27,\n",
       " 42,\n",
       " 203,\n",
       " 39,\n",
       " 39,\n",
       " 31,\n",
       " 79,\n",
       " 42,\n",
       " 160,\n",
       " 37,\n",
       " 94,\n",
       " 41,\n",
       " 174,\n",
       " 55,\n",
       " 126,\n",
       " 26,\n",
       " 141,\n",
       " 60,\n",
       " 40,\n",
       " 156,\n",
       " 49,\n",
       " 21,\n",
       " 40,\n",
       " 22,\n",
       " 82,\n",
       " 16,\n",
       " 57,\n",
       " 48,\n",
       " 248,\n",
       " 46,\n",
       " 63,\n",
       " 46,\n",
       " 117,\n",
       " 63,\n",
       " 40,\n",
       " 52,\n",
       " 26,\n",
       " 72,\n",
       " 196,\n",
       " 80,\n",
       " 116,\n",
       " 28,\n",
       " 52,\n",
       " 14,\n",
       " 36,\n",
       " 122,\n",
       " 97,\n",
       " 36,\n",
       " 54,\n",
       " 28,\n",
       " 15,\n",
       " 137,\n",
       " 45,\n",
       " 93,\n",
       " 64,\n",
       " 84,\n",
       " 20,\n",
       " 136,\n",
       " 34,\n",
       " 52,\n",
       " 46,\n",
       " 220,\n",
       " 28,\n",
       " 51,\n",
       " 56,\n",
       " 41,\n",
       " 39,\n",
       " 22,\n",
       " 123,\n",
       " 22,\n",
       " 58,\n",
       " 67,\n",
       " 89,\n",
       " 68,\n",
       " 35,\n",
       " 66,\n",
       " 33,\n",
       " 41,\n",
       " 39,\n",
       " 75,\n",
       " 18,\n",
       " 39,\n",
       " 40,\n",
       " 63,\n",
       " 35,\n",
       " 123,\n",
       " 87,\n",
       " 169,\n",
       " 21,\n",
       " 20,\n",
       " 54,\n",
       " 61,\n",
       " 64,\n",
       " 48,\n",
       " 61,\n",
       " 101,\n",
       " 50,\n",
       " 40,\n",
       " 45,\n",
       " 102,\n",
       " 18,\n",
       " 41,\n",
       " 44,\n",
       " 17,\n",
       " 49,\n",
       " 187,\n",
       " 45,\n",
       " 51,\n",
       " 67,\n",
       " 70,\n",
       " 107,\n",
       " 50,\n",
       " 117,\n",
       " 401,\n",
       " 15,\n",
       " 20,\n",
       " 28,\n",
       " 93,\n",
       " 28,\n",
       " 58,\n",
       " 23,\n",
       " 72,\n",
       " 33,\n",
       " 90,\n",
       " 44,\n",
       " 48,\n",
       " 44,\n",
       " 95,\n",
       " 58,\n",
       " 44,\n",
       " 64,\n",
       " 55,\n",
       " 45,\n",
       " 50,\n",
       " 80,\n",
       " 39,\n",
       " 77,\n",
       " 116,\n",
       " 130,\n",
       " 77,\n",
       " 70,\n",
       " 33,\n",
       " 57,\n",
       " 42,\n",
       " 63,\n",
       " 56,\n",
       " 153,\n",
       " 27,\n",
       " 14,\n",
       " 217,\n",
       " 98,\n",
       " 43,\n",
       " 102,\n",
       " 52,\n",
       " 31,\n",
       " 37,\n",
       " 76,\n",
       " 27,\n",
       " 48,\n",
       " 118,\n",
       " 15,\n",
       " 181,\n",
       " 65,\n",
       " 102,\n",
       " 66,\n",
       " 54,\n",
       " 32,\n",
       " 196,\n",
       " 86,\n",
       " 51,\n",
       " 21,\n",
       " 57,\n",
       " 95,\n",
       " 141,\n",
       " 39,\n",
       " 37,\n",
       " 118,\n",
       " 165,\n",
       " 15,\n",
       " 108,\n",
       " 56,\n",
       " 29,\n",
       " 42,\n",
       " 91,\n",
       " 14,\n",
       " 20,\n",
       " 33,\n",
       " 87,\n",
       " 43,\n",
       " 105,\n",
       " 220,\n",
       " 48,\n",
       " 102,\n",
       " 31,\n",
       " 46,\n",
       " 32,\n",
       " 64,\n",
       " 57,\n",
       " 39,\n",
       " 39,\n",
       " 14,\n",
       " 223,\n",
       " 17,\n",
       " 52,\n",
       " 175,\n",
       " 40,\n",
       " 20,\n",
       " 68,\n",
       " 31,\n",
       " 48,\n",
       " 49,\n",
       " 146,\n",
       " 32,\n",
       " 34,\n",
       " 40,\n",
       " 11,\n",
       " 48,\n",
       " 61,\n",
       " 118,\n",
       " 27,\n",
       " 13,\n",
       " 64,\n",
       " 176,\n",
       " 32,\n",
       " 99,\n",
       " 32,\n",
       " 136,\n",
       " 59,\n",
       " 38,\n",
       " 34,\n",
       " 134,\n",
       " 49,\n",
       " 79,\n",
       " 42,\n",
       " 115,\n",
       " 46,\n",
       " 58,\n",
       " 37,\n",
       " 21,\n",
       " 32,\n",
       " 89,\n",
       " 83,\n",
       " 31,\n",
       " 76,\n",
       " 117,\n",
       " 37,\n",
       " 22,\n",
       " 39,\n",
       " 17,\n",
       " 118,\n",
       " 75,\n",
       " 36,\n",
       " 34,\n",
       " 86,\n",
       " 96,\n",
       " 20,\n",
       " 54,\n",
       " 57,\n",
       " 28,\n",
       " 55,\n",
       " 44,\n",
       " 44,\n",
       " 15,\n",
       " 10,\n",
       " 58,\n",
       " 94,\n",
       " 32,\n",
       " 31,\n",
       " 42,\n",
       " 61,\n",
       " 26,\n",
       " 81,\n",
       " 14,\n",
       " 55,\n",
       " 52,\n",
       " 110,\n",
       " 26,\n",
       " 47,\n",
       " 41,\n",
       " 58,\n",
       " 56,\n",
       " 120,\n",
       " 40,\n",
       " 52,\n",
       " 140,\n",
       " 92,\n",
       " 69,\n",
       " 77,\n",
       " 125,\n",
       " 49,\n",
       " 82,\n",
       " 58,\n",
       " 50,\n",
       " 102,\n",
       " 37,\n",
       " 123,\n",
       " 16,\n",
       " 280,\n",
       " 82,\n",
       " 40,\n",
       " 18,\n",
       " 289,\n",
       " 89,\n",
       " 44,\n",
       " 82,\n",
       " 33,\n",
       " 58,\n",
       " 39,\n",
       " 173,\n",
       " 102,\n",
       " 67,\n",
       " 106,\n",
       " 22,\n",
       " 40,\n",
       " 14,\n",
       " 113,\n",
       " 87,\n",
       " 94,\n",
       " 259,\n",
       " 57,\n",
       " 24,\n",
       " 74,\n",
       " 33,\n",
       " 132,\n",
       " 111,\n",
       " 72,\n",
       " 100,\n",
       " 117,\n",
       " 169,\n",
       " 190,\n",
       " 11,\n",
       " 43,\n",
       " 22,\n",
       " 53,\n",
       " 59,\n",
       " 63,\n",
       " 66,\n",
       " 14,\n",
       " 75,\n",
       " 57,\n",
       " 101,\n",
       " 248,\n",
       " 63,\n",
       " 40,\n",
       " 142,\n",
       " 175,\n",
       " 21,\n",
       " 31,\n",
       " 71,\n",
       " 47,\n",
       " 79,\n",
       " 55,\n",
       " 40,\n",
       " 36,\n",
       " 48,\n",
       " 171,\n",
       " 66,\n",
       " 96,\n",
       " 115,\n",
       " 99,\n",
       " 68,\n",
       " 173,\n",
       " 28,\n",
       " 43,\n",
       " 275,\n",
       " 58,\n",
       " 204,\n",
       " 59,\n",
       " 104,\n",
       " 63,\n",
       " 15,\n",
       " 70,\n",
       " 121,\n",
       " 16,\n",
       " 78,\n",
       " 26,\n",
       " 104,\n",
       " 37,\n",
       " 58,\n",
       " 53,\n",
       " 76,\n",
       " 26,\n",
       " 49,\n",
       " 149,\n",
       " 25,\n",
       " 163,\n",
       " 52,\n",
       " 70,\n",
       " 18,\n",
       " 63,\n",
       " 46,\n",
       " 123,\n",
       " 54,\n",
       " 108,\n",
       " 37,\n",
       " 84,\n",
       " 40,\n",
       " 19,\n",
       " 34,\n",
       " 141,\n",
       " 45,\n",
       " 22,\n",
       " 86,\n",
       " 53,\n",
       " 122,\n",
       " 71,\n",
       " 117,\n",
       " 33,\n",
       " 50,\n",
       " 34,\n",
       " 75,\n",
       " 40,\n",
       " 44,\n",
       " 99,\n",
       " 22,\n",
       " 44,\n",
       " 85,\n",
       " 61,\n",
       " 98,\n",
       " 76,\n",
       " 80,\n",
       " 35,\n",
       " 117,\n",
       " 75,\n",
       " 50,\n",
       " 57,\n",
       " 33,\n",
       " 118,\n",
       " 28,\n",
       " 83,\n",
       " 103,\n",
       " 87,\n",
       " 296,\n",
       " 131,\n",
       " 33,\n",
       " 40,\n",
       " 39,\n",
       " 32,\n",
       " 66,\n",
       " 38,\n",
       " 134,\n",
       " 18,\n",
       " 92,\n",
       " 63,\n",
       " 36,\n",
       " 42,\n",
       " 104,\n",
       " 17,\n",
       " 77,\n",
       " 110,\n",
       " 40,\n",
       " 57,\n",
       " 52,\n",
       " 49,\n",
       " 96,\n",
       " 25,\n",
       " 60,\n",
       " 105,\n",
       " 151,\n",
       " 14,\n",
       " 26,\n",
       " 70,\n",
       " 32,\n",
       " 55,\n",
       " 86,\n",
       " 91,\n",
       " 91,\n",
       " 28,\n",
       " 104,\n",
       " 122,\n",
       " 82,\n",
       " 120,\n",
       " 47,\n",
       " 70,\n",
       " 85,\n",
       " 39,\n",
       " 40,\n",
       " 38,\n",
       " 40,\n",
       " 121,\n",
       " 40,\n",
       " 21,\n",
       " 83,\n",
       " 16,\n",
       " 91,\n",
       " 283,\n",
       " 58,\n",
       " 39,\n",
       " 55,\n",
       " 48,\n",
       " 137,\n",
       " 58,\n",
       " 76,\n",
       " 89,\n",
       " 39,\n",
       " 67,\n",
       " 32,\n",
       " 139,\n",
       " 234,\n",
       " 41,\n",
       " 20,\n",
       " 28,\n",
       " 49,\n",
       " 85,\n",
       " 48,\n",
       " 56,\n",
       " 16,\n",
       " 15,\n",
       " 45,\n",
       " 77,\n",
       " 61,\n",
       " 11,\n",
       " 38,\n",
       " 54,\n",
       " 125,\n",
       " 21,\n",
       " 112,\n",
       " 62,\n",
       " 167,\n",
       " 14,\n",
       " 74,\n",
       " 26,\n",
       " 58,\n",
       " 44,\n",
       " 57,\n",
       " 55,\n",
       " 32,\n",
       " 60,\n",
       " 114,\n",
       " 121,\n",
       " 51,\n",
       " 56,\n",
       " 82,\n",
       " 78,\n",
       " 113,\n",
       " 21,\n",
       " 139,\n",
       " 95,\n",
       " 88,\n",
       " 14,\n",
       " 79,\n",
       " 83,\n",
       " 165,\n",
       " 14,\n",
       " 38,\n",
       " 66,\n",
       " 38,\n",
       " 59,\n",
       " 47,\n",
       " 118,\n",
       " 297,\n",
       " 56,\n",
       " 14,\n",
       " 99,\n",
       " 113,\n",
       " 61,\n",
       " 62,\n",
       " 35,\n",
       " 68,\n",
       " 63,\n",
       " 232,\n",
       " 28,\n",
       " 44,\n",
       " 51,\n",
       " 142,\n",
       " 39,\n",
       " 54,\n",
       " 49,\n",
       " 15,\n",
       " 49,\n",
       " 58,\n",
       " 28,\n",
       " 53,\n",
       " 29,\n",
       " 54,\n",
       " 137,\n",
       " 91,\n",
       " 270,\n",
       " 68,\n",
       " 22,\n",
       " 154,\n",
       " 19,\n",
       " 9,\n",
       " 68,\n",
       " 17,\n",
       " 19,\n",
       " 50,\n",
       " 36,\n",
       " 42,\n",
       " 59,\n",
       " 117,\n",
       " 139,\n",
       " 72,\n",
       " 45,\n",
       " 46,\n",
       " 73,\n",
       " 52,\n",
       " 21,\n",
       " 28,\n",
       " 101,\n",
       " 99,\n",
       " 37,\n",
       " 64,\n",
       " 42,\n",
       " 115,\n",
       " 77,\n",
       " 142,\n",
       " 75,\n",
       " 13,\n",
       " 26,\n",
       " 86,\n",
       " 65,\n",
       " 23,\n",
       " 46,\n",
       " 70,\n",
       " 49,\n",
       " 48,\n",
       " 28,\n",
       " 42,\n",
       " 424,\n",
       " 116,\n",
       " 52,\n",
       " 55,\n",
       " 66,\n",
       " 78,\n",
       " 91,\n",
       " 105,\n",
       " 42,\n",
       " 81,\n",
       " 122,\n",
       " 66,\n",
       " 65,\n",
       " 68,\n",
       " 69,\n",
       " 104,\n",
       " 28,\n",
       " 57,\n",
       " 109,\n",
       " 30,\n",
       " 20,\n",
       " 57,\n",
       " 67,\n",
       " 55,\n",
       " 16,\n",
       " 37,\n",
       " 63,\n",
       " 31,\n",
       " 65,\n",
       " 76,\n",
       " 97,\n",
       " 30,\n",
       " 34,\n",
       " 54,\n",
       " 53,\n",
       " 90,\n",
       " 83,\n",
       " 142,\n",
       " 48,\n",
       " 55,\n",
       " 67,\n",
       " 15,\n",
       " 42,\n",
       " 72,\n",
       " 57,\n",
       " 31,\n",
       " 131,\n",
       " 65,\n",
       " 105,\n",
       " 88,\n",
       " 88,\n",
       " 151,\n",
       " 52,\n",
       " 98,\n",
       " 34,\n",
       " 289,\n",
       " 127,\n",
       " 78,\n",
       " 67,\n",
       " 55,\n",
       " 115,\n",
       " 90,\n",
       " 28,\n",
       " 103,\n",
       " 74,\n",
       " 175,\n",
       " 42,\n",
       " 44,\n",
       " 53,\n",
       " 44,\n",
       " 31,\n",
       " 52,\n",
       " 125,\n",
       " 75,\n",
       " 99,\n",
       " 94,\n",
       " 69,\n",
       " 127,\n",
       " 180,\n",
       " 84,\n",
       " 66,\n",
       " 9,\n",
       " 87,\n",
       " 58,\n",
       " 45,\n",
       " 132,\n",
       " 55,\n",
       " 62,\n",
       " 47,\n",
       " 48,\n",
       " 56,\n",
       " 31,\n",
       " 39,\n",
       " 17,\n",
       " 37,\n",
       " 72,\n",
       " 259,\n",
       " 276,\n",
       " 206,\n",
       " 66,\n",
       " 32,\n",
       " 42,\n",
       " 116,\n",
       " 47,\n",
       " 90,\n",
       " 76,\n",
       " 35,\n",
       " 41,\n",
       " 89,\n",
       " 100,\n",
       " 15,\n",
       " 98,\n",
       " 28,\n",
       " 28,\n",
       " 78,\n",
       " 161,\n",
       " 80,\n",
       " 34,\n",
       " 70,\n",
       " 43,\n",
       " 53,\n",
       " 40,\n",
       " 116,\n",
       " 27,\n",
       " 140,\n",
       " 77,\n",
       " 40,\n",
       " 120,\n",
       " 22,\n",
       " 87,\n",
       " 102,\n",
       " 82,\n",
       " 40,\n",
       " 118,\n",
       " 16,\n",
       " 47,\n",
       " 18,\n",
       " 25,\n",
       " 48,\n",
       " 87,\n",
       " 71,\n",
       " 87,\n",
       " 40,\n",
       " 76,\n",
       " 73,\n",
       " 17,\n",
       " 42,\n",
       " 105,\n",
       " 54,\n",
       " 37,\n",
       " 91,\n",
       " 25,\n",
       " 22,\n",
       " 87,\n",
       " 118,\n",
       " 72,\n",
       " 9,\n",
       " 38,\n",
       " 42,\n",
       " 40,\n",
       " 58,\n",
       " 33,\n",
       " 94,\n",
       " 175,\n",
       " 51,\n",
       " 100,\n",
       " 46,\n",
       " 14,\n",
       " 35,\n",
       " 46,\n",
       " 213,\n",
       " 126,\n",
       " 80,\n",
       " 132,\n",
       " 52,\n",
       " 79,\n",
       " 15,\n",
       " 54,\n",
       " 24,\n",
       " 72,\n",
       " 49,\n",
       " 301,\n",
       " 99,\n",
       " 60,\n",
       " 74,\n",
       " 132,\n",
       " 28,\n",
       " 29,\n",
       " 33,\n",
       " 32,\n",
       " 137,\n",
       " 32,\n",
       " 38,\n",
       " 47,\n",
       " 82,\n",
       " 132,\n",
       " 401,\n",
       " 35,\n",
       " 84,\n",
       " 77,\n",
       " 30,\n",
       " 18,\n",
       " ...]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_formula_len_dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "29df030e-5eeb-4bd3-bb5e-1447a80c2c85",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_formula_len_sorted = np.argsort(train_formula_len_dist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "32e0805e-59d4-4721-bb6b-f47227560ea3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29837"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_formula_len_sorted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "97f753c2-f36b-4d65-b8ec-19e97ae8130d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sqrt(6)'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[train_formula_len_sorted[0]]['annotated_formula']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7216bd7c-e95f-492f-9c72-c1089a8e383d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sqrt(54)'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[train_formula_len_sorted[10]]['annotated_formula']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "b181af7c-aa1c-4e13-a8d8-858a01d707ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'add(5, 3)'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[train_formula_len_sorted[100]]['annotated_formula']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "9c9d9e59-61ed-4d0d-b4a5-a254d939682b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'multiply(10, 5)'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[train_formula_len_sorted[1000]]['annotated_formula']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "215518a9-bbe9-4b00-9240-a6e0ceb39edb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_step(sent):\n",
    "    doc = nlp(sent)\n",
    "    n_step = len(list(s for s in doc.sents))\n",
    "    return n_step\n",
    "\n",
    "def get_ans_dict(c):\n",
    "    if(c.startswith(\"['\")):\n",
    "        c_ = eval(c)\n",
    "        ans_dict = {}\n",
    "        for ac in c_:\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "    else:\n",
    "        ans_dict = {}\n",
    "        pattern = ' , [a-e] \\) '\n",
    "        c_ = re.split(pattern, c)\n",
    "        c_[0] = c_[0].split(' ) ')[-1]\n",
    "        for i, c in zip(['a', 'b', 'c', 'd', 'e'], c_):\n",
    "            ans_dict[i] = c\n",
    "    return ans_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6987518b-c0e7-448b-8060-65b1c7387677",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_answer(pred_str, ans_str):\n",
    "    \"\"\"Find the last number as the predicted answer\"\"\"\n",
    "    pattern = '\\d*\\.?\\d+'\n",
    "    pred = re.findall(pattern, pred_str)\n",
    "    if(len(pred) >= 1):\n",
    "        # print(pred_str)\n",
    "        pred = pred[-1]\n",
    "        gold = re.findall(pattern, ans_str)\n",
    "        # print(ans_str)\n",
    "        gold = gold[-1]\n",
    "        return pred == gold\n",
    "    else: return False\n",
    "\n",
    "def parse_pred_ans(filename):\n",
    "    with open(filename) as fd: lines = fd.readlines()\n",
    "    am, a = None, None\n",
    "    num_q, acc = 0, 0\n",
    "    current_mode = 'none'\n",
    "    questions = []\n",
    "    ans_pred = []\n",
    "    ans_gold = []\n",
    "    for l in lines:\n",
    "        if(l.startswith('Question: ')):\n",
    "            if(current_mode != 'none' and current_mode != 'a'):\n",
    "                # print(l)\n",
    "                current_mode = 'none'\n",
    "                continue\n",
    "            if(am is not None and a is not None):\n",
    "                questions.append(q)\n",
    "                ans_pred.append(am)\n",
    "                ans_gold.append(a)\n",
    "                if(test_answer(am, a)):\n",
    "                    acc += 1\n",
    "            current_mode = 'q'\n",
    "            q = l\n",
    "            num_q += 1\n",
    "        elif(l.startswith('Options: ')): pass\n",
    "        elif(l.startswith('A_model:')):\n",
    "            current_mode = 'am'\n",
    "            am = l\n",
    "        elif(l.startswith('A:')):\n",
    "            current_mode = 'a'\n",
    "            a = l\n",
    "        else:\n",
    "            if(current_mode == 'q'): q += l\n",
    "            elif(current_mode == 'am'): am += l\n",
    "            elif(current_mode == 'a'): a += l\n",
    "            elif(current_mode == 'none'): pass\n",
    "            else:\n",
    "                raise ValueError(current_mode)\n",
    "                \n",
    "    questions.append(q)\n",
    "    ans_pred.append(am)\n",
    "    ans_gold.append(a)\n",
    "    if(test_answer(am, a)):\n",
    "        acc += 1\n",
    "    print('num_q %d correct %d ratio %.4f' % (num_q, acc, float(acc / num_q)))\n",
    "    return questions, ans_pred, ans_gold"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a09a6852-907f-4089-861a-641fb7f79581",
   "metadata": {},
   "source": [
    "# Shortest Formula"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "ce7cd214-81a9-4bec-a915-9cc5d2d35fd8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.5\n"
     ]
    }
   ],
   "source": [
    "prompt_short_formula = ''\n",
    "\n",
    "q_prev = ''\n",
    "i = 0\n",
    "formula_len = []\n",
    "for idx in train_formula_len_sorted[0:1000]:\n",
    "    q = train_data[idx]['Problem']\n",
    "    if(q != q_prev):\n",
    "        c = train_data[idx]['options']\n",
    "        cr = train_data[idx]['correct']\n",
    "        r = train_data[idx]['Rationale']\n",
    "\n",
    "        ans_dict = get_ans_dict(c)\n",
    "\n",
    "        if(r[0] == '\"'): r = r[1:]\n",
    "        if(r[-1] == '\"'): r = r[:-1]\n",
    "\n",
    "        r += ' ' + ans_dict[cr]\n",
    "\n",
    "        prompt_short_formula += 'Question: ' + q + '\\n'\n",
    "        prompt_short_formula += 'Options: ' + c + '\\n'\n",
    "        prompt_short_formula += \"Let's think step by step\\n\"\n",
    "        prompt_short_formula += r + '\\n\\n'\n",
    "        q_prev = q\n",
    "        formula_len.append(len(train_data[idx]['annotated_formula']))\n",
    "        i += 1\n",
    "        if(i == 8): break\n",
    "print(np.average(formula_len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "39e4d742-97a9-48da-8d95-33863693a645",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: what is the range of all the roots of | x ^ 2 - 6 | = x ?\n",
      "Options: a ) 4 , b ) 3 , c ) 2 , d ) 1 , e ) 5\n",
      "Let's think step by step\n",
      "we get 2 quadratic equations here . . 1 ) x ^ 2 - x - 2 = 0 . . . . . . . roots 2 , - 1 2 ) x ^ 2 + x - 2 = 0 . . . . . . . . roots - 2 , 1 inserting each root in given equation , it can be seen that - 1 and - 2 do not satisfy the equations . so value of x for given equation . . . . x = 6 or x = 1 i guess range is 6 - 1 = 5 e 5\n",
      "\n",
      "Question: what is the range of all the roots of | x ^ 2 - 1 | = x ?\n",
      "Options: a ) 4 , b ) 3 , c ) 2 , d ) 1 , e ) 0\n",
      "Let's think step by step\n",
      "we get 2 quadratic equations here . . 1 ) x ^ 2 - x - 1 = 0 . . . . . . . roots 2 , - 1 2 ) x ^ 2 + x - 1 = 0 . . . . . . . . roots - 2 , 1 inserting each root in given equation , it can be seen that - 1 and - 2 do not satisfy the equations . so value of x for given equation . . . . x = 2 or x = 1 i guess range is 2 - 1 = 1 d 1\n",
      "\n",
      "Question: two right circular cylinders of equal volumes have their heights in the ratio 1 : 2 . find the ratio of their radii .\n",
      "Options: ['a ) √ 3 : 1', 'b ) √ 7 : 1', 'c ) √ 2 : 1', 'd ) 2 : 1', 'e ) 3 : 1']\n",
      "Let's think step by step\n",
      "explanation : let their heights be h and 2 h and radii be r and r respectively then . π r 2 h = π r 2 ( 2 h ) = > r 2 / r 2 = 2 h / h = 2 / 1 = > r / r = √ 2 / 1 = > r : r = √ 2 : 1 answer is c √ 2 : 1\n",
      "\n",
      "Question: what is the range of all the roots of | x ^ 2 - 5 | = x ?\n",
      "Options: a ) 4 , b ) 3 , c ) 2 , d ) 1 , e ) 0\n",
      "Let's think step by step\n",
      "we get 2 quadratic equations here . . 1 ) x ^ 2 - x - 5 = 0 . . . . . . . roots 2 , - 1 2 ) x ^ 2 + x - 5 = 0 . . . . . . . . roots - 2 , 1 inserting each root in given equation , it can be seen that - 1 and - 2 do not satisfy the equations . so value of x for given equation . . . . x = 5 or x = 1 i guess range is 5 - 1 = 4 a 4\n",
      "\n",
      "Question: from below option 49 is divisible by which one ?\n",
      "Options: a ) a ) 2 , b ) b ) 3 , c ) c ) 4 , d ) d ) 5 , e ) e ) 7\n",
      "Let's think step by step\n",
      "49 / 7 = 7 e e ) 7\n",
      "\n",
      "Question: the length of the longest tape in cm which can be used to measure exactly , the length 10 m ; 3 m 85 cm ; and 11 m 50 cm is :\n",
      "Options: a ) 5 , b ) 35 , c ) 34 , d ) 36 , e ) 38\n",
      "Let's think step by step\n",
      "the three lengths in cm are 1000 , 385 & 1150 . hcf of 700 , 385 & 1295 is 5 . hence , the answer is 5 cm . answer : a 5\n",
      "\n",
      "Question: from below option 27 is divisible by which one ?\n",
      "Options: a ) a ) 4 , b ) b ) 8 , c ) c ) 9 , d ) d ) 10 , e ) e ) 911\n",
      "Let's think step by step\n",
      "27 / 9 = 3 c c ) 9\n",
      "\n",
      "Question: the length of the longest tape in cm which can be used to measure exactly , the length 100 cm ; 2 m 25 cm ; and 7 m 80 cm is :\n",
      "Options: a ) 30 , b ) 5 , c ) 10 , d ) 36 , e ) 25\n",
      "Let's think step by step\n",
      "the three lengths in cm are 100 , 225 & 780 . hcf of 100 , 225 & 780 is 5 . hence , the answer is 5 cm . answer : b 5\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_short_formula)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "6e749f2b-9ef8-47a4-81f8-5795939dd965",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_short_formula = open('MathQA/prompt_short_formula.txt').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "978d3feb-07fc-47f0-b3f9-d7625682a163",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████| 200/200 [14:25<00:00,  4.33s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_short_formula.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "        # for ac in c.split(', '):\n",
    "        #     idx_, num_ = ac.split(' ) ')[-2:]\n",
    "        #     if('a' in idx_): ans_dict['a'] = num_\n",
    "        #     if('b' in idx_): ans_dict['b'] = num_\n",
    "        #     if('c' in idx_): ans_dict['c'] = num_\n",
    "        #     if('d' in idx_): ans_dict['d'] = num_\n",
    "        #     if('e' in idx_): ans_dict['e'] = num_\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = prompt_short_formula + '\\n' + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "d12b68a1-c7cc-4375-bd9a-548807506234",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 75 ratio 0.3750\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_short_formula.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fad6ffe-8162-477a-b29b-b7c4df5f2ff9",
   "metadata": {},
   "source": [
    "# Mid-Length Formula"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "ed1e13a7-4f85-4285-8936-9bd5c063d4f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55.0\n"
     ]
    }
   ],
   "source": [
    "prompt_mid_length_formula = ''\n",
    "\n",
    "q_prev = ''\n",
    "i = 0\n",
    "formula_len = []\n",
    "for idx in train_formula_len_sorted[14000:15000]:\n",
    "    q = train_data[idx]['Problem']\n",
    "    if(q != q_prev):\n",
    "        c = train_data[idx]['options']\n",
    "        cr = train_data[idx]['correct']\n",
    "        r = train_data[idx]['Rationale']\n",
    "\n",
    "        ans_dict = get_ans_dict(c)\n",
    "\n",
    "        if(r[0] == '\"'): r = r[1:]\n",
    "        if(r[-1] == '\"'): r = r[:-1]\n",
    "\n",
    "        r += ' ' + ans_dict[cr]\n",
    "\n",
    "        prompt_mid_length_formula += 'Question: ' + q + '\\n'\n",
    "        prompt_mid_length_formula += 'Options: ' + c + '\\n'\n",
    "        prompt_mid_length_formula += \"Let's think step by step\\n\"\n",
    "        prompt_mid_length_formula += r + '\\n\\n'\n",
    "        q_prev = q\n",
    "        formula_len.append(len(train_data[idx]['annotated_formula']))\n",
    "        i += 1\n",
    "        if(i == 8): break\n",
    "print(np.average(formula_len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "e78f9ebd-052c-4b49-90dc-ed33de666537",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: if 20 men can build a water fountain 56 metres long in 21 days , what length of a similar water fountain can be built by 35 men in 3 days ?\n",
      "Options: a ) 10 m , b ) 14 m , c ) 17 m , d ) 19 m , e ) 20 m\n",
      "Let's think step by step\n",
      "explanation : let the required length be x metres more men , more length built ( direct proportion ) less days , less length built ( direct proportion ) men 20 : 35 days 21 : 3 : : 56 : x therefore ( 20 x 21 x x ) = ( 35 x 3 x 56 ) x = ( 35 x 3 x 56 ) / 420 = 14 hence , the required length is 14 m . answer : b 14 m\n",
      "\n",
      "Question: a train , 125 meters long travels at a speed of 45 km / hr crosses a bridge in 30 seconds . the length of the bridge is\n",
      "Options: a ) 250 m , b ) 245 m , c ) 235 m , d ) 220 m , e ) 240 m\n",
      "Let's think step by step\n",
      "explanation : assume the length of the bridge = x meter total distance covered = 125 + x meter total time taken = 30 s speed = total distance covered / total time taken = ( 125 + x ) / 30 m / s = > 45 ã — ( 10 / 36 ) = ( 125 + x ) / 30 = > 45 ã — 10 ã — 30 / 36 = 125 + x = > 45 ã — 10 ã — 10 / 12 = 125 + x = > 15 ã — 10 ã — 10 / 4 = 125 + x = > 15 ã — 25 = 125 + x = 375 = > x = 375 - 125 = 250 answer : option a 250 m\n",
      "\n",
      "Question: a train 250 m long running at 72 kmph crosses a platform in 40 sec . what is the length of the platform ?\n",
      "Options: a ) 450 m , b ) 200 m , c ) 250 m , d ) 270 m , e ) 300 m\n",
      "Let's think step by step\n",
      "d = 72 * 5 / 18 = 40 = 800 â € “ 250 = 450 m answer : a 450 m\n",
      "\n",
      "Question: if 30 % of a number is equal to one - third of another number , what is the ratio of first number to the second number ?\n",
      "Options: a ) 2 : 5 , b ) 3 : 7 , c ) 10 : 9 , d ) 7 : 3 , e ) none of these\n",
      "Let's think step by step\n",
      "explanation : let 30 % of a = 1 / 3 b then , 30 a / 100 = 1 b / 3 = > 3 a / 10 = b / 3 = > a : b = 10 : 9 answer : c 10 : 9\n",
      "\n",
      "Question: a train 100 m long can cross an electric pole in 20 sec and then find the speed of the train ?\n",
      "Options: a ) 19 kmph , b ) 18 kmph , c ) 72 kmph , d ) 17 kmph , e ) 91 kmph\n",
      "Let's think step by step\n",
      "length = speed * time speed = l / t s = 100 / 20 s = 5 m / sec speed = 5 * 18 / 5 ( to convert m / sec in to kmph multiply by 18 / 5 ) speed = 18 kmph answer : b 18 kmph\n",
      "\n",
      "Question: a man walking at the rate of 6 km / hr crosses a bridge in 15 minutes . the length of the bridge is ______ .\n",
      "Options: a ) 1000 m , b ) 1250 m , c ) 1500 m , d ) 1800 m , e ) 2300 m\n",
      "Let's think step by step\n",
      "hint : to find the answer in meter , we will first convert distance from km / hour to meter / sec by multiplying it with 5 / 18 . also , change 15 minutes to seconds by multiplying it with 60 . distance = speed x time 1 . convert speed into m / sec : 6 x 5 / 18 m / s = 1.66 m / s 2 . convert time from minutes into seconds = 15 x 60 s = 900 sec 3 . calculate : distance = 1.66 x 900 = 1500 m answer is c 1500 m\n",
      "\n",
      "Question: the cost price of a radio is rs . 1890 and it was sold for rs . 1500 , find the loss % ?\n",
      "Options: a ) 18 , b ) 20 , c ) 77 , d ) 66 , e ) 41\n",
      "Let's think step by step\n",
      "1890 - - - - 390 100 - - - - ? = > 20 % answer : b 20\n",
      "\n",
      "Question: it takes joey the postman 1 hours to run a 5 mile long route every day . he delivers packages and then returns to the post office along the same path . if the average speed of the round trip is 8 mile / hour , what is the speed with which joey returns ?\n",
      "Options: a ) 11 , b ) 12 , c ) 13 , d ) 14 , e ) 20\n",
      "Let's think step by step\n",
      "let his speed for one half of the journey be 5 miles an hour let the other half be x miles an hour now , avg speed = 8 mile an hour 2 * 5 * x / 5 + x = 8 10 x = 8 x + 40 = > 2 x = 40 = > x = 20 e 20\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_mid_length_formula)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "e52c23d0-fd38-4596-a6d5-b768152f5665",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_mid_length_formula = open('MathQA/prompt_mid_length_formula.txt').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "5179b619-2637-4370-96c7-3adc87792203",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████| 200/200 [16:44<00:00,  5.02s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_mid_length_formula.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = prompt_mid_length_formula + '\\n' + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "be56b5fb-c310-4a3c-a56f-be47ca6c54f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 67 ratio 0.3350\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_mid_length_formula.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "816bc164-a744-4a68-8b3c-61a8b9d45ae2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "09f8602e-a9f0-40f4-925b-f3add079f0f1",
   "metadata": {},
   "source": [
    "# Longest Formula"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "f77488bb-bcb9-450e-9cb9-c4c5285d7c40",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "165.0\n"
     ]
    }
   ],
   "source": [
    "prompt_long_formula = ''\n",
    "\n",
    "q_prev = ''\n",
    "i = 0\n",
    "formula_len = []\n",
    "for idx in train_formula_len_sorted[28000:]:\n",
    "    q = train_data[idx]['Problem']\n",
    "    if(q != q_prev):\n",
    "        c = train_data[idx]['options']\n",
    "        cr = train_data[idx]['correct']\n",
    "        r = train_data[idx]['Rationale']\n",
    "\n",
    "        ans_dict = get_ans_dict(c)\n",
    "\n",
    "        if(r[0] == '\"'): r = r[1:]\n",
    "        if(r[-1] == '\"'): r = r[:-1]\n",
    "\n",
    "        r += ' ' + ans_dict[cr]\n",
    "\n",
    "        prompt_long_formula += 'Question: ' + q + '\\n'\n",
    "        prompt_long_formula += 'Options: ' + c + '\\n'\n",
    "        prompt_long_formula += \"Let's think step by step\\n\"\n",
    "        prompt_long_formula += r + '\\n\\n'\n",
    "        q_prev = q\n",
    "        i += 1\n",
    "        formula_len.append(len(train_data[idx]['annotated_formula']))\n",
    "        if(i == 20): break\n",
    "print(np.average(formula_len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "c4cd308f-0deb-4723-acf5-81234c06b40f",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: a $ 500 investment and a $ 1,500 investment have a combined yearly return of 16 percent of the total of the two investments . if the $ 500 investment has a yearly return of 7 percent , what percent yearly return does the $ 1,500 investment have ?\n",
      "Options: a ) 9 % , b ) 10 % , c ) 105 / 8 % , d ) 11 % , e ) 19 %\n",
      "Let's think step by step\n",
      "the equation we can form the question : return on total investment = sum of individual investments ( 500 + 1500 ) ( 16 ) = ( 500 â ˆ — 7 ) + ( 1500 x ) , where x is the return on investment of 1500 . solving the equation , we get x = 19 % ( option e ) answer : e 19 %\n",
      "\n",
      "Question: john distributes his pencil among his 4 friends rose , mary , ranjan , and rohit in the ratio 1 / 2 : 1 / 3 : 1 / 3 : 1 / 5 . what is the minimum no . of pencils that the person should have ?\n",
      "Options: a ) 41 , b ) 65 , c ) 70 , d ) 77 , e ) 98\n",
      "Let's think step by step\n",
      "rakesh : rahul : ranjan : rohit = 1 / 2 : 1 / 3 : 1 / 3 : 1 / 5 step 1 : at first we need to do is lcm of 2 , 3,3 and 5 is 30 . step 2 : then pencil are distributed in ratio among friends , rakesh = ( 1 / 2 x 30 ) = 15 . rahul = ( 1 / 3 x 30 ) = 10 . ranjan = ( 1 / 3 x 30 ) = 10 . rohit = ( 1 / 5 x 30 ) = 6 . step 3 : total number of pencils are ( 15 x + 10 x + 10 x + 6 x ) = 41 x . for minimum number of pencils x = 1 . the person should have at least 41 pencils . a ) 41\n",
      "\n",
      "Question: how much more would rs . 50000 fetch , after two years , if it is put at 20 % p . a . compound interest payable half yearly than if is put at 20 % p . a . compound interest payable yearly ?\n",
      "Options: a ) 482 , b ) 1205 , c ) 992 , d ) 837 , e ) 923\n",
      "Let's think step by step\n",
      "50000 ( 11 / 10 ) 4 - 50000 ( 6 / 5 ) 2 = 1205 answer : b 1205\n",
      "\n",
      "Question: on a certain day , tim invested $ 600 at 10 percent annual interest , compounded annually , and lana invested 1,000 at 5 percent annual interest , compounded annually . the total amount of interest earned by tim ’ s investment in the first 2 years was how much greater than the total amount of interest earned by lana ’ s investment in the first 2 years ?\n",
      "Options: a ) $ 5.5 , b ) $ 15.5 , c ) $ 23.5 , d ) $ 100 , e ) $ 105\n",
      "Let's think step by step\n",
      "compounded annually means that the interest is applied once per year . one can have 10 % annual interest compounded monthly - in this case 10 % / 12 would be applied each month , or 10 % annual interest compounded daily etc . with respect to the problem at hand , at the end of two years , tim would have 600 ( 1.10 ) ^ 2 = 600 ( 1.21 ) = 726 and lana would have 1,000 ( 1.05 ) ^ 2 = 1,000 ( 1.1025 ) = 1102.5 thus , tim earned 126 dollars , while lana earned 102.5 dollars the difference is $ 23.5 and the answer is c . $ 23.5\n",
      "\n",
      "Question: a $ 500 investment and a $ 1,500 investment have a combined yearly return of 22 percent of the total of the two investments . if the $ 500 investment has a yearly return of 7 percent , what percent yearly return does the $ 1,500 investment have ?\n",
      "Options: a ) 29 % , b ) 20 % , c ) 27 % , d ) 25 % , e ) 12 %\n",
      "Let's think step by step\n",
      "the equation we can form the question : return on total investment = sum of individual investments ( 500 + 1500 ) ( 22 ) = ( 500 â ˆ — 7 ) + ( 1500 x ) , where x is the return on investment of 1500 . solving the equation , we get x = 27 % ( option c ) answer : c 27 %\n",
      "\n",
      "Question: john distributes his pencil among his 4 friends rose , mary , ranjan , and rohit in the ratio 1 / 1 : 1 / 3 : 1 / 4 : 1 / 5 . what is the minimum no . of pencils that the person should have ?\n",
      "Options: a ) 45 , b ) 65 , c ) 98 , d ) 107 , e ) 198\n",
      "Let's think step by step\n",
      "rakesh : rahul : ranjan : rohit = 1 / 1 : 1 / 3 : 1 / 4 : 1 / 5 step 1 : at first we need to do is lcm of 2 , 3,4 and 5 is 60 . step 2 : then pencil are distributed in ratio among friends , rakesh = ( 1 / 1 x 60 ) = 60 . rahul = ( 1 / 3 x 60 ) = 20 . ranjan = ( 1 / 4 x 60 ) = 15 . rohit = ( 1 / 5 x 60 ) = 12 . step 3 : total number of pencils are ( 60 x + 20 x + 15 x + 12 x ) = 107 x . for minimum number of pencils x = 1 . the person should have at least 107 pencils . d ) 107\n",
      "\n",
      "Question: during a sale of 30 % on everything in a store , a kid is successful in convincing the store manager to give him 20 candies for the discounted price of 14 candies . the store still makes a profit of 12 % on this sale . what is the mark up percentage on each candy ?\n",
      "Options: a ) 100 % , b ) 80 % , c ) 75 % , d ) 66 + 2 / 3 % , e ) 55 %\n",
      "Let's think step by step\n",
      "hi akhil , i can share the way i solved it . . let ' s say marked price = m so , there ' s a discount of 20 % on m so , new s . p . = 80 % of m now , the child convinces the owner to sell 20 candies for the price of 14 candies . let ' s say each candy after discount is 1 $ . so , s . p . of 20 candies = 20 $ . the child bought it for 14 $ so , he got a discount of 6 / 20 * 100 = 30 % so , the latest s . p . = 70 % of 80 % of m = 0.7 * 0.8 m now , we are given that the shopkeeper still makes a profit of 12 % . so we have , 0.7 * 0.8 * m = 1.12 c . p so , we get , m = 2 c . p . i . e . marked price was kept 100 % above c . p . b 80 %\n",
      "\n",
      "Question: how much more would rs . 20000 fetch , after two years , if it is put at 20 % p . a . compound interest payable half yearly than if is put at 20 % p . a . compound interest payable yearly ?\n",
      "Options: a ) a ) rs . 6 , 602.50 , b ) b ) rs . 6 , 601.50 , c ) c ) rs . 6 , 603.50 , d ) d ) rs . 6 , 604.50 , e ) e ) rs . 6 , 605.50\n",
      "Let's think step by step\n",
      "20000 ( 11 / 10 ) 4 - 20000 ( 6 / 5 ) 2 = 6 , 602.50 answer : a rs . 6 , 602.50\n",
      "\n",
      "Question: a $ 500 investment and a $ 1,500 investment have a combined yearly return of 10 percent of the total of the two investments . if the $ 500 investment has a yearly return of 7 percent , what percent yearly return does the $ 1,500 investment have ?\n",
      "Options: a ) 9 % , b ) 10 % , c ) 105 / 8 % , d ) 11 % , e ) 12 %\n",
      "Let's think step by step\n",
      "the equation we can form the question : return on total investment = sum of individual investments ( 500 + 1500 ) ( 10 ) = ( 500 â ˆ — 7 ) + ( 1500 x ) , where x is the return on investment of 1500 . solving the equation , we get x = 11 % ( option d ) answer : d 11 %\n",
      "\n",
      "Question: a $ 500 investment and a $ 1,500 investment have a combined yearly return of 19 percent of the total of the two investments . if the $ 500 investment has a yearly return of 7 percent , what percent yearly return does the $ 1,500 investment have ?\n",
      "Options: a ) 9 % , b ) 10 % , c ) 23 % , d ) 21 % , e ) 22 %\n",
      "Let's think step by step\n",
      "the equation we can form the question : return on total investment = sum of individual investments ( 500 + 1500 ) ( 19 ) = ( 500 â ˆ — 7 ) + ( 1500 x ) , where x is the return on investment of 1500 . solving the equation , we get x = 23 % ( option c ) answer : c 23 %\n",
      "\n",
      "Question: during a sale of 20 % on everything in a store , a kid is successful in convincing the store manager to give him 40 candies for the discounted price of 14 candies . the store still makes a profit of 12 % on this sale . what is the mark up percentage on each candy ?\n",
      "Options: a ) 100 % , b ) 80 % , c ) 75 % , d ) 66 + 2 / 3 % , e ) 55 %\n",
      "Let's think step by step\n",
      "hi akhil , i can share the way i solved it . . let ' s say marked price = m so , there ' s a discount of 20 % on m so , new s . p . = 80 % of m now , the child convinces the owner to sell 20 candies for the price of 14 candies . let ' s say each candy after discount is 1 $ . so , s . p . of 20 candies = 20 $ . the child bought it for 14 $ so , he got a discount of 6 / 20 * 100 = 30 % so , the latest s . p . = 70 % of 80 % of m = 0.7 * 0.8 m now , we are given that the shopkeeper still makes a profit of 12 % . so we have , 0.7 * 0.8 * m = 1.12 c . p so , we get , m = 2 c . p . i . e . marked price was kept 100 % above c . p . e 55 %\n",
      "\n",
      "Question: susan made a block with small cubes of 8 cubic cm volume to make a block , 3 small cubes long , 9 small cubes wide and 5 small cubes deep . she realizes that she has used more small cubes than she really needed . she realized that she could have glued a fewer number of cubes together to lock like a block with same dimensions , if it were made hollow . what is the minimum number of cubes that she needs to make the block ?\n",
      "Options: ['a ) 113', 'b ) 114', 'c ) 115', 'd ) 116', 'e ) 117']\n",
      "Let's think step by step\n",
      "the total volume ( in terms of number of cubes ) of the solid = 3 * 9 * 5 = 135 the total volume ( in terms of number of cubes ) of the hollow = ( 3 - 2 ) * ( 9 - 2 ) * ( 5 - 2 ) = 21 so number of cubes required = 135 - 21 = 114 answer : b 114\n",
      "\n",
      "Question: the length of a rectangular plot is 10 mtr more than its width . the cost of fencing the plot along its perimeter at the rate of rs . 6.5 mtr is rs . 2210 . the perimeter of the plot is ?\n",
      "Options: a ) 126 , b ) 156 , c ) 340 , d ) 321 , e ) 260\n",
      "Let's think step by step\n",
      "sol . let width = x , length = ( 10 + x ) perimeter = 2 ( x + ( 10 + x ) ) = 2 ( 2 x = 10 ) & 2 ( 2 x + 10 ) * 6.5 = 2210 x = 80 required perimeter = 2 ( 80 + 90 ) = 340 c 340\n",
      "\n",
      "Question: a $ 500 investment and a $ 1,500 investment have a combined yearly return of 13 percent of the total of the two investments . if the $ 500 investment has a yearly return of 7 percent , what percent yearly return does the $ 1,500 investment have ?\n",
      "Options: a ) 9 % , b ) 15 % , c ) 105 / 8 % , d ) 11 % , e ) 12 %\n",
      "Let's think step by step\n",
      "the equation we can form the question : return on total investment = sum of individual investments ( 500 + 1500 ) ( 13 ) = ( 500 â ˆ — 7 ) + ( 1500 x ) , where x is the return on investment of 1500 . solving the equation , we get x = 15 % ( option b ) answer : b 15 %\n",
      "\n",
      "Question: what is the smallest 5 digit number that is divisible by 15 , 32 , 45 , and 54 ?\n",
      "Options: a ) 11260 , b ) 11860 , c ) 12360 , d ) 12960 , e ) 13560\n",
      "Let's think step by step\n",
      "15 = 3 * 5 32 = 2 ^ 5 45 = 3 ^ 2 * 5 54 = 2 * 3 ^ 3 lcm = 2 ^ 5 * 3 ^ 3 * 5 = 4320 the smallest five - digit number that is a multiple of 4320 is 3 * 4320 = 12,960 the answer is d . 12960\n",
      "\n",
      "Question: when a mobile is sold for rs . 24000 , the owner loses 40 % . at what price must that mobile be sold in order to gain 40 % ?\n",
      "Options: a ) 54,000 , b ) 56,000 , c ) 58,000 , d ) 60,000 , e ) 62,000\n",
      "Let's think step by step\n",
      "60 : 24000 = 140 : x x = ( 24000 x 140 ) / 60 = 56000 . hence , s . p . = rs . 56,000 . answer : option b 56,000\n",
      "\n",
      "Question: how much more would rs . 60000 fetch , after two years , if it is put at 20 % p . a . compound interest payable half yearly than if is put at 20 % p . a . compound interest payable yearly ?\n",
      "Options: a ) 482 , b ) 725 , c ) 992 , d ) 1446 , e ) 923\n",
      "Let's think step by step\n",
      "60000 ( 11 / 10 ) 4 - 60000 ( 6 / 5 ) 2 = 1446 answer : d 1446\n",
      "\n",
      "Question: a man rows 750 m in 675 seconds against the stream and returns in 7 and half minutes . his rowing speed in s ɵ ll water is\n",
      "Options: a ) 4 kmph , b ) 5 kmph , c ) 6 kmph , d ) 7 kmph , e ) none of these\n",
      "Let's think step by step\n",
      "explanation : rate upstream = ( 750 / 675 ) = 10 / 9 m / sec rate downstream ( 750 / 450 ) m / sec = 5 / 3 m / sec rate in still water = ( 1 / 2 ) * [ ( 10 / 9 ) + ( 5 / 3 ) ] m / sec . = 25 / 18 m / sec = ( 25 / 18 ) * ( 18 / 5 ) kmph = 5 kmph answer : b 5 kmph\n",
      "\n",
      "Question: during a sale of 20 % on everything in a store , a kid is successful in convincing the store manager to give him 20 candies for the discounted price of 14 candies . the store still makes a profit of 10 % on this sale . what is the mark up percentage on each candy ?\n",
      "Options: a ) 100 % , b ) 80 % , c ) 75 % , d ) 66 + 2 / 3 % , e ) 55 %\n",
      "Let's think step by step\n",
      "hi akhil , i can share the way i solved it . . let ' s say marked price = m so , there ' s a discount of 20 % on m so , new s . p . = 80 % of m now , the child convinces the owner to sell 20 candies for the price of 14 candies . let ' s say each candy after discount is 1 $ . so , s . p . of 20 candies = 20 $ . the child bought it for 14 $ so , he got a discount of 6 / 20 * 100 = 30 % so , the latest s . p . = 70 % of 80 % of m = 0.7 * 0.8 m now , we are given that the shopkeeper still makes a profit of 12 % . so we have , 0.7 * 0.8 * m = 1.12 c . p so , we get , m = 2 c . p . i . e . marked price was kept 100 % above c . p . c 75 %\n",
      "\n",
      "Question: john distributes his pencil among his 4 friends rose , mary , ranjan , and rohit in the ratio 1 / 5 : 1 / 3 : 1 / 4 : 1 / 5 . what is the minimum no . of pencils that the person should have ?\n",
      "Options: a ) 59 , b ) 65 , c ) 70 , d ) 77 , e ) 98\n",
      "Let's think step by step\n",
      "rakesh : rahul : ranjan : rohit = 1 / 5 : 1 / 3 : 1 / 4 : 1 / 5 step 1 : at first we need to do is lcm of 2 , 3,4 and 5 is 60 . step 2 : then pencil are distributed in ratio among friends , rakesh = ( 1 / 5 x 60 ) = 12 rahul = ( 1 / 3 x 60 ) = 20 . ranjan = ( 1 / 4 x 60 ) = 15 . rohit = ( 1 / 5 x 60 ) = 12 . step 3 : total number of pencils are ( 12 x + 20 x + 15 x + 12 x ) = 59 x . for minimum number of pencils x = 1 . the person should have at least 59 pencils . a ) 59\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_long_formula)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "8fd2ef6b-34d9-44cb-852d-5d8eafcf3253",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_long_formula = open('MathQA/prompt_long_formula.txt').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "265be494-23ee-44e3-8c44-1b8a80a89495",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(prompt_long_formula.split('\\n')) - 5 * 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "e18b7eca-1977-4931-a7ec-9686511cf9c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████| 200/200 [13:26<00:00,  4.03s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_long_formula.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += 'Question: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = prompt_long_formula + '\\n' + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "2fe17a98-4c21-447a-8427-1651e390aa55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 87 ratio 0.4350\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_long_formula.txt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
